1 Modeltime Forecasting

1.1 options & settings

For scrollable output

options(scipen = 999)

1.3 install packages

# install.packages("modeltime")

1.4 Load libs

library(tidyverse)
library(tidymodels)
library(modeltime)
library(timetk)
library(lubridate)

1.5 Load data

read.csv("day.csv") %>%  head()
head(bike_sharing_daily)
bike_transactions_tbl <- bike_sharing_daily %>% 
                          select(dteday, cnt) %>% 
                          set_names(c("date","value"))

head(bike_transactions_tbl)

1.5.1 Visualizing loaded data

bike_transactions_tbl %>% 
  timetk::plot_time_series(date, value)


#  timetk::plot_time_series(date, value, .interactive = FALSE)

1.6 Train Test

  • Setting assess = "3 months" tells the function to use the last 3-months of data as the testing set.
  • Setting cumulative = TRUE tells the sampling to use all of the prior data as the training set.
splits <- bike_transactions_tbl %>% 
            time_series_split(assess = "3 months", cumulative = TRUE)

1.6.1 Visulaizing Train test split


splits %>% 
  timetk::tk_time_series_cv_plan() %>% 
  timetk::plot_time_series_cv_plan(date, value)

1.7 Modeling

1.7.1 Automatic Models

1.7.1.1 Auto Arima

model_fit_arima <- modeltime::arima_reg() %>% 
                    parsnip::set_engine("auto_arima") %>% 
                    parsnip::fit(value ~ date, training(splits))

model_fit_arima
parsnip model object

Fit time:  5.8s 
Series: outcome 
ARIMA(0,1,3) with drift 

Coefficients:
          ma1      ma2      ma3   drift
      -0.6106  -0.1868  -0.0673  9.3169
s.e.   0.0396   0.0466   0.0398  4.6225

sigma^2 estimated as 730568:  log likelihood=-5227.22
AIC=10464.44   AICc=10464.53   BIC=10486.74

1.7.1.2 Prophet

model_fit_prophet <- prophet_reg() %>%
                      set_engine("prophet", yearly.seasonality = TRUE) %>%
                      fit(value ~ date, training(splits))

model_fit_prophet
parsnip model object

Fit time:  1.4s 
PROPHET Model
- growth: 'linear'
- n.changepoints: 25
- changepoint.range: 0.8
- yearly.seasonality: 'auto'
- weekly.seasonality: 'auto'
- daily.seasonality: 'auto'
- seasonality.mode: 'additive'
- changepoint.prior.scale: 0.05
- seasonality.prior.scale: 10
- holidays.prior.scale: 10
- logistic_cap: NULL
- logistic_floor: NULL
- extra_regressors: 0

1.7.2 ML Models

Machine learning models are more complex than the automated models. This complexity typically requires a workflow (sometimes called a pipeline in other languages). The general process goes like this:

  • Create Preprocessing Recipe

  • Create Model Specifications

  • Use Workflow to combine Model Spec and Preprocessing, and Fit Model

1.7.2.1 Preprocessing Recipe

First, I’ll create a preprocessing recipe using recipe() and adding time series steps. The process uses the “date” column to create 45 new features that I’d like to model. These include time-series signature features and fourier series.

recipe_spec <- recipe(value ~ date, training(splits)) %>% 
                step_timeseries_signature(date) %>% 
                step_rm(contains("am.pm"), contains("hour"), contains("minute"),
                        contains("second"), contains("xts")) %>% 
                step_fourier(date, period = 365, K = 5) %>% 
                step_dummy(all_nominal())

recipe_spec %>% 
  prep() %>% 
  juice()

With a recipe in-hand, we can set up our machine learning pipelines.

1.7.2.2 Elastic Net / glmnet

model_spec_glmnet <- linear_reg(penalty = 0.01, mixture = 0.5) %>% 
                      set_engine("glmnet")

model_spec_glmnet
Linear Regression Model Specification (regression)

Main Arguments:
  penalty = 0.01
  mixture = 0.5

Computational engine: glmnet 
workflow_fit_glmnet <- workflow() %>% 
  add_model(model_spec_glmnet) %>% 
  add_recipe(recipe_spec %>% step_rm(date)) %>% 
  fit(training(splits))

workflow_fit_glmnet
== Workflow [trained] ==========================================================
Preprocessor: Recipe
Model: linear_reg()

-- Preprocessor ----------------------------------------------------------------
5 Recipe Steps

* step_timeseries_signature()
* step_rm()
* step_fourier()
* step_dummy()
* step_rm()

-- Model -----------------------------------------------------------------------

Call:  glmnet::glmnet(x = as.matrix(x), y = y, family = "gaussian",      alpha = ~0.5) 

   Df  %Dev  Lambda
1   0  0.00 2856.00
2   1  5.71 2602.00
3   2 11.22 2371.00
4   2 19.05 2160.00
5   3 26.52 1968.00
6   3 33.31 1793.00
7   3 39.30 1634.00
8   4 44.58 1489.00
9   4 49.32 1357.00
10  5 53.41 1236.00
11  5 56.95 1126.00
12  5 59.98 1026.00
13  5 62.58  935.10
14  5 64.81  852.00
15  5 66.71  776.30
16  5 68.34  707.30
17  5 69.72  644.50
18  5 70.91  587.20
19  5 71.91  535.10
20  5 72.77  487.50
21  6 73.52  444.20
22  6 74.19  404.80
23  6 74.77  368.80
24  6 75.25  336.00
25  6 75.65  306.20
26  6 75.99  279.00
27  7 76.35  254.20
28  7 76.68  231.60
29  8 76.99  211.00
30  8 77.26  192.30
31 10 77.51  175.20
32 10 77.74  159.60
33 10 77.93  145.50
34 10 78.09  132.50
35 11 78.24  120.80
36 12 78.36  110.00
37 13 78.47  100.30
38 14 78.58   91.36
39 14 78.66   83.24
40 15 78.75   75.85
41 15 78.81   69.11
42 15 78.87   62.97
43 17 78.93   57.37
44 18 78.99   52.28
45 18 79.05   47.63
46 18 79.09   43.40

...
and 42 more lines.

1.7.2.3 RandomForest

model_spec_rf <- rand_forest(trees = 500, min_n = 50) %>% 
                  set_engine("randomForest")

workflow_fit_rf <- workflow() %>% 
  add_model(model_spec_rf) %>% 
  add_recipe(recipe_spec %>% step_rm(date)) %>%
  fit(training(splits))

workflow_fit_rf
== Workflow [trained] ==========================================================
Preprocessor: Recipe
Model: rand_forest()

-- Preprocessor ----------------------------------------------------------------
5 Recipe Steps

* step_timeseries_signature()
* step_rm()
* step_fourier()
* step_dummy()
* step_rm()

-- Model -----------------------------------------------------------------------

Call:
 randomForest(x = as.data.frame(x), y = y, ntree = ~500, nodesize = ~50) 
               Type of random forest: regression
                     Number of trees: 500
No. of variables tried at each split: 15

          Mean of squared residuals: 701813.6
                    % Var explained: 80.94

1.7.3 New Hybrid Models

1.7.3.1 Prophet Boost

The Prophet Boost algorithm combines Prophet with XGBoost to get the best of both worlds (i.e. Prophet Automation + Machine Learning). The algorithm works by:

  1. First modeling the univariate series using Prophet

  2. Using regressors supplied via the preprocessing recipe (remember our recipe generated 45 new features), and regressing the Prophet Residuals with the XGBoost model

We can set the model up using a workflow just like with the machine learning algorithms.

model_spec_prophet_boost <- prophet_boost() %>% 
                              set_engine("prophet_xgboost", yearly.seasonality = TRUE)

workflow_fit_prophet_boost <- workflow() %>% 
  add_model(model_spec_prophet_boost) %>% 
  add_recipe(recipe_spec) %>% 
  fit(training(splits))

1.8 Modeltime Table

model_table <- modeltime_table(
                  model_fit_arima,
                  model_fit_prophet,
                  workflow_fit_glmnet,
                  workflow_fit_rf,
                  workflow_fit_prophet_boost
                  )

model_table
# Modeltime Table

1.9 Claibration

calibration_table <- model_table %>% 
  modeltime_calibrate(testing(splits))

calibration_table
# Modeltime Table

1.10 Forecasting

library(plotly)
calibration_table %>% 
  modeltime_forecast(actual_data = bike_transactions_tbl) %>% 
  plot_modeltime_forecast() %>% 
  layout(legend = list(x = -0.5, y = 1.0))


  # for more legend position options check out https://plotly.com/r/legend/
P_results1 <- calibration_table %>% 
  modeltime_forecast(actual_data = bike_transactions_tbl) %>% 
  plot_modeltime_forecast() %>% 
  layout(legend = list(orientation = "h", y = -0.2))
Using '.calibration_data' to forecast.
P_results1
library(htmlwidgetsl)
Error in library(htmlwidgetsl) : 
  there is no package called ‘htmlwidgetsl’
htmlwidgets::saveWidget(P_results1, "forecasting_results.html")

1.11 Accuracy (Testing set)

calibration_table %>% 
  modeltime_accuracy() %>% 
  table_modeltime_accuracy(.interactive = FALSE)
Accuracy Table
.model_id .model_desc .type mae mape mase smape rmse rsq
1 ARIMA(0,1,3) WITH DRIFT Test 2540.11 474.89 2.74 46.00 3188.09 0.39
2 PROPHET Test 3052.52 513.91 3.30 51.51 3707.56 0.29
3 GLMNET Test 1243.53 332.96 1.34 29.20 1693.58 0.48
4 RANDOMFOREST Test 1335.74 334.77 1.44 30.60 1851.42 0.48
5 PROPHET W/ XGBOOST ERRORS Test 2528.39 412.40 2.73 46.02 3139.75 0.07

1.12 Refit and Forecast Forward

calibration_table %>% 
  
  # Remove arima model with low accuracy
  filter(.model_id !=1) %>% 
  
  # Refit & Forecast Forward
  modeltime_refit(bike_transactions_tbl) %>% 
  modeltime_forecast(h = "12 months", actual_data = bike_transactions_tbl) %>%
  plot_modeltime_forecast() %>% 
  layout(legend = list(orientation = "h", y = -0.2))
LS0tDQp0aXRsZTogIk1vZGVsdGltZSBGb3JlY2FzdGluZyByLWJsb2dnZXJzIg0Kb3V0cHV0OiANCiAgaHRtbF9ub3RlYm9vazoNCiAgICBoaWdobGlnaHQ6IHRhbmdvDQogICAgZGZfcHJpbnQ6IHBhZ2VkDQogICAgdG9jOiB0cnVlDQogICAgdG9jX2Zsb2F0OiANCiAgICAgIGNvbGxhcHNlZDogZmFsc2UNCiAgICAgIHNtb290aF9zY3JvbGw6IGZhbHNlDQogICAgbnVtYmVyX3NlY3Rpb25zOiB0cnVlDQogICAgdG9jX2RlcHRoOiA2DQotLS0NCg0KDQojIE1vZGVsdGltZSBGb3JlY2FzdGluZw0KDQojIyBvcHRpb25zICYgc2V0dGluZ3MNCg0KRm9yIHNjcm9sbGFibGUgb3V0cHV0DQoNCmBgYHtjc3MsIGVjaG89RkFMU0V9DQouc2Nyb2xsLTEwMCB7DQogIG1heC1oZWlnaHQ6IDEwMHB4Ow0KICBvdmVyZmxvdy15OiBhdXRvOw0KICBiYWNrZ3JvdW5kLWNvbG9yOiBpbmhlcml0Ow0KfQ0KDQoNCmgxLCAjVE9DPnVsPmxpIHsNCiAgY29sb3I6ICNCNjREM0E7DQp9DQoNCmgyLCAjVE9DPnVsPnVsPmxpIHsNCiAgY29sb3I6ICMwMDAwMDA7DQp9DQoNCmgzLCAjVE9DPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICM2NDNjYjI7DQp9DQoNCmg0LCAjVE9DPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNhZTAwNTg7DQp9DQoNCmg1LCAjVE9DPnVsPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNmZmE0NDc7DQp9DQoNCmg2LCAjVE9DPnVsPnVsPnVsPnVsPnVsPnVsPmxpIHsNCiAgY29sb3I6ICNEQUUzRDk7DQp9DQoNCmBgYA0KDQoNCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFfQ0Ka25pdHI6Om9wdHNfY2h1bmskc2V0KGVjaG8gPSBUUlVFLCBtZXNzYWdlID0gRkFMU0UsIHdhcm5pbmcgPSBGQUxTRSwgZHBpID0gMzAwICxhdHRyLm91dHB1dD0nc3R5bGU9Im1heC1oZWlnaHQ6IDMwMHB4OyInLCBvdXQud2lkdGggPSAiMTAwJSIsIGZpZy53aWR0aD0xMikNCmBgYA0KDQoNCmBgYHtyfQ0Kb3B0aW9ucyhzY2lwZW4gPSA5OTkpDQpgYGANCg0KDQojIyBmcm9tDQoNCmh0dHBzOi8vd3d3LmJ1c2luZXNzLXNjaWVuY2UuaW8vY29kZS10b29scy8yMDIwLzA2LzI5L2ludHJvZHVjaW5nLW1vZGVsdGltZS5odG1sDQpodHRwczovL3d3dy5yLWJsb2dnZXJzLmNvbS8yMDIwLzA2L2ludHJvZHVjaW5nLW1vZGVsdGltZS10aWR5LXRpbWUtc2VyaWVzLWZvcmVjYXN0aW5nLXVzaW5nLXRpZHltb2RlbHMvDQoNCg0KKipiaWtlIHNoYXJpbmcgZGF0YSBmcm9tKioNCg0KQXZhaWxhYmxlIHdpdGhpbiBsaWJyYXJ5IGJ5IG5hbWUgYGJpa2Vfc2hhcmluZ19kYWlseWAgb3IgZG93bmxvYWQgZnJvbSBiZWxvdyBsaW5rDQoNCmh0dHBzOi8vYXJjaGl2ZS5pY3MudWNpLmVkdS9tbC9tYWNoaW5lLWxlYXJuaW5nLWRhdGFiYXNlcy8wMDI3NS8NCg0KKipSZWZlcmVuY2Ugb2Ygc29tZSBFREEgb24gQmlrZSBzaGFyaW5nIGRhdGEgKioNCg0KaHR0cHM6Ly93d3cucnB1YnMuY29tL3NoYXlpbmkvYmlrZV9zaGFyaW5nDQpodHRwOi8vcnN0dWRpby1wdWJzLXN0YXRpYy5zMy5hbWF6b25hd3MuY29tLzE1ODU5NV8xZjUyMGZkOGQ4ZTM0YTVhYjNhMTI3Mzc2ZjJmNjE2OS5odG1sDQoNCg0KIyMgaW5zdGFsbCBwYWNrYWdlcw0KDQpgYGB7ciB9DQojIGluc3RhbGwucGFja2FnZXMoIm1vZGVsdGltZSIpDQpgYGANCg0KIyMgTG9hZCBsaWJzDQoNCmBgYHtyIH0NCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeSh0aWR5bW9kZWxzKQ0KbGlicmFyeShtb2RlbHRpbWUpDQpsaWJyYXJ5KHRpbWV0aykNCmxpYnJhcnkobHVicmlkYXRlKQ0KYGBgDQoNCiMjIExvYWQgZGF0YQ0KDQpgYGB7cn0NCnJlYWQuY3N2KCJkYXkuY3N2IikgJT4lICBoZWFkKCkNCmBgYA0KDQoNCmBgYHtyfQ0KaGVhZChiaWtlX3NoYXJpbmdfZGFpbHkpDQpgYGANCg0KDQpgYGB7cn0NCmJpa2VfdHJhbnNhY3Rpb25zX3RibCA8LSBiaWtlX3NoYXJpbmdfZGFpbHkgJT4lIA0KICAgICAgICAgICAgICAgICAgICAgICAgICBzZWxlY3QoZHRlZGF5LCBjbnQpICU+JSANCiAgICAgICAgICAgICAgICAgICAgICAgICAgc2V0X25hbWVzKGMoImRhdGUiLCJ2YWx1ZSIpKQ0KDQpoZWFkKGJpa2VfdHJhbnNhY3Rpb25zX3RibCkNCmBgYA0KDQoNCiMjIyBWaXN1YWxpemluZyBsb2FkZWQgZGF0YQ0KDQpgYGB7cn0NCmJpa2VfdHJhbnNhY3Rpb25zX3RibCAlPiUgDQogIHRpbWV0azo6cGxvdF90aW1lX3NlcmllcyhkYXRlLCB2YWx1ZSkNCg0KIyAgdGltZXRrOjpwbG90X3RpbWVfc2VyaWVzKGRhdGUsIHZhbHVlLCAuaW50ZXJhY3RpdmUgPSBGQUxTRSkNCmBgYA0KDQojIyBUcmFpbiBUZXN0DQoNCg0KICAtIFNldHRpbmcgYGFzc2VzcyA9ICIzIG1vbnRocyJgIHRlbGxzIHRoZSBmdW5jdGlvbiB0byB1c2UgdGhlIGxhc3QgMy1tb250aHMgb2YgZGF0YSBhcyB0aGUgdGVzdGluZyBzZXQuDQogIC0gU2V0dGluZyBgY3VtdWxhdGl2ZSA9IFRSVUVgIHRlbGxzIHRoZSBzYW1wbGluZyB0byB1c2UgYWxsIG9mIHRoZSBwcmlvciBkYXRhIGFzIHRoZSB0cmFpbmluZyBzZXQuDQoNCg0KYGBge3J9DQpzcGxpdHMgPC0gYmlrZV90cmFuc2FjdGlvbnNfdGJsICU+JSANCiAgICAgICAgICAgIHRpbWVfc2VyaWVzX3NwbGl0KGFzc2VzcyA9ICIzIG1vbnRocyIsIGN1bXVsYXRpdmUgPSBUUlVFKQ0KYGBgDQoNCiMjIyBWaXN1bGFpemluZyBUcmFpbiB0ZXN0IHNwbGl0DQoNCmBgYHtyfQ0KDQpzcGxpdHMgJT4lIA0KICB0aW1ldGs6OnRrX3RpbWVfc2VyaWVzX2N2X3BsYW4oKSAlPiUgDQogIHRpbWV0azo6cGxvdF90aW1lX3Nlcmllc19jdl9wbGFuKGRhdGUsIHZhbHVlKQ0KYGBgDQoNCiMjIE1vZGVsaW5nDQoNCiMjIyBBdXRvbWF0aWMgTW9kZWxzDQoNCiMjIyMgQXV0byBBcmltYQ0KDQpgYGB7cn0NCm1vZGVsX2ZpdF9hcmltYSA8LSBtb2RlbHRpbWU6OmFyaW1hX3JlZygpICU+JSANCiAgICAgICAgICAgICAgICAgICAgcGFyc25pcDo6c2V0X2VuZ2luZSgiYXV0b19hcmltYSIpICU+JSANCiAgICAgICAgICAgICAgICAgICAgcGFyc25pcDo6Zml0KHZhbHVlIH4gZGF0ZSwgdHJhaW5pbmcoc3BsaXRzKSkNCg0KbW9kZWxfZml0X2FyaW1hDQpgYGANCg0KIyMjIyBQcm9waGV0DQoNCmBgYHtyfQ0KbW9kZWxfZml0X3Byb3BoZXQgPC0gcHJvcGhldF9yZWcoKSAlPiUNCiAgICAgICAgICAgICAgICAgICAgICBzZXRfZW5naW5lKCJwcm9waGV0IiwgeWVhcmx5LnNlYXNvbmFsaXR5ID0gVFJVRSkgJT4lDQogICAgICAgICAgICAgICAgICAgICAgZml0KHZhbHVlIH4gZGF0ZSwgdHJhaW5pbmcoc3BsaXRzKSkNCg0KbW9kZWxfZml0X3Byb3BoZXQNCmBgYA0KDQojIyMgTUwgTW9kZWxzDQoNCg0KTWFjaGluZSBsZWFybmluZyBtb2RlbHMgYXJlIG1vcmUgY29tcGxleCB0aGFuIHRoZSBhdXRvbWF0ZWQgbW9kZWxzLiBUaGlzIGNvbXBsZXhpdHkgdHlwaWNhbGx5IHJlcXVpcmVzIGEgKip3b3JrZmxvdyoqIChzb21ldGltZXMgY2FsbGVkIGEgcGlwZWxpbmUgaW4gb3RoZXIgbGFuZ3VhZ2VzKS4gVGhlIGdlbmVyYWwgcHJvY2VzcyBnb2VzIGxpa2UgdGhpczoNCg0KICAtICoqQ3JlYXRlIFByZXByb2Nlc3NpbmcgUmVjaXBlKioNCiAgDQogIC0gKipDcmVhdGUgTW9kZWwgU3BlY2lmaWNhdGlvbnMqKg0KICANCiAgLSAqKlVzZSBXb3JrZmxvdyB0byBjb21iaW5lIE1vZGVsIFNwZWMgYW5kIFByZXByb2Nlc3NpbmcsIGFuZCBGaXQgTW9kZWwqKg0KDQoNCiMjIyMgUHJlcHJvY2Vzc2luZyBSZWNpcGUNCg0KRmlyc3QsIEnigJlsbCBjcmVhdGUgYSBwcmVwcm9jZXNzaW5nIHJlY2lwZSB1c2luZyBgcmVjaXBlKClgIGFuZCBhZGRpbmcgdGltZSBzZXJpZXMgc3RlcHMuIFRoZSBwcm9jZXNzIHVzZXMgdGhlIGDigJxkYXRl4oCdYCBjb2x1bW4gdG8gY3JlYXRlIGA0NSBuZXcgZmVhdHVyZXNgIHRoYXQgSeKAmWQgbGlrZSB0byBtb2RlbC4gVGhlc2UgaW5jbHVkZSB0aW1lLXNlcmllcyBzaWduYXR1cmUgZmVhdHVyZXMgYW5kIGZvdXJpZXIgc2VyaWVzLg0KDQpgYGB7cn0NCnJlY2lwZV9zcGVjIDwtIHJlY2lwZSh2YWx1ZSB+IGRhdGUsIHRyYWluaW5nKHNwbGl0cykpICU+JSANCiAgICAgICAgICAgICAgICBzdGVwX3RpbWVzZXJpZXNfc2lnbmF0dXJlKGRhdGUpICU+JSANCiAgICAgICAgICAgICAgICBzdGVwX3JtKGNvbnRhaW5zKCJhbS5wbSIpLCBjb250YWlucygiaG91ciIpLCBjb250YWlucygibWludXRlIiksDQogICAgICAgICAgICAgICAgICAgICAgICBjb250YWlucygic2Vjb25kIiksIGNvbnRhaW5zKCJ4dHMiKSkgJT4lIA0KICAgICAgICAgICAgICAgIHN0ZXBfZm91cmllcihkYXRlLCBwZXJpb2QgPSAzNjUsIEsgPSA1KSAlPiUgDQogICAgICAgICAgICAgICAgc3RlcF9kdW1teShhbGxfbm9taW5hbCgpKQ0KDQpyZWNpcGVfc3BlYyAlPiUgDQogIHByZXAoKSAlPiUgDQogIGp1aWNlKCkNCmBgYA0KDQoNCldpdGggYSByZWNpcGUgaW4taGFuZCwgd2UgY2FuIHNldCB1cCBvdXIgbWFjaGluZSBsZWFybmluZyBwaXBlbGluZXMuDQoNCg0KIyMjIyBFbGFzdGljIE5ldCAvIGdsbW5ldA0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfZ2xtbmV0IDwtIGxpbmVhcl9yZWcocGVuYWx0eSA9IDAuMDEsIG1peHR1cmUgPSAwLjUpICU+JSANCiAgICAgICAgICAgICAgICAgICAgICBzZXRfZW5naW5lKCJnbG1uZXQiKQ0KDQptb2RlbF9zcGVjX2dsbW5ldA0KYGBgDQoNCmBgYHtyfQ0Kd29ya2Zsb3dfZml0X2dsbW5ldCA8LSB3b3JrZmxvdygpICU+JSANCiAgYWRkX21vZGVsKG1vZGVsX3NwZWNfZ2xtbmV0KSAlPiUgDQogIGFkZF9yZWNpcGUocmVjaXBlX3NwZWMgJT4lIHN0ZXBfcm0oZGF0ZSkpICU+JSANCiAgZml0KHRyYWluaW5nKHNwbGl0cykpDQoNCndvcmtmbG93X2ZpdF9nbG1uZXQNCmBgYA0KDQojIyMjIFJhbmRvbUZvcmVzdA0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfcmYgPC0gcmFuZF9mb3Jlc3QodHJlZXMgPSA1MDAsIG1pbl9uID0gNTApICU+JSANCiAgICAgICAgICAgICAgICAgIHNldF9lbmdpbmUoInJhbmRvbUZvcmVzdCIpDQoNCndvcmtmbG93X2ZpdF9yZiA8LSB3b3JrZmxvdygpICU+JSANCiAgYWRkX21vZGVsKG1vZGVsX3NwZWNfcmYpICU+JSANCiAgYWRkX3JlY2lwZShyZWNpcGVfc3BlYyAlPiUgc3RlcF9ybShkYXRlKSkgJT4lDQogIGZpdCh0cmFpbmluZyhzcGxpdHMpKQ0KDQp3b3JrZmxvd19maXRfcmYNCmBgYA0KDQojIyMgTmV3IEh5YnJpZCBNb2RlbHMNCg0KIyMjIyBQcm9waGV0IEJvb3N0DQoNClRoZSBQcm9waGV0IEJvb3N0IGFsZ29yaXRobSBjb21iaW5lcyBQcm9waGV0IHdpdGggWEdCb29zdCB0byBnZXQgdGhlIGJlc3Qgb2YgYm90aCB3b3JsZHMgKGkuZS4gUHJvcGhldCBBdXRvbWF0aW9uICsgTWFjaGluZSBMZWFybmluZykuIFRoZSBhbGdvcml0aG0gd29ya3MgYnk6DQoNCiAgMS4gRmlyc3QgbW9kZWxpbmcgdGhlIHVuaXZhcmlhdGUgc2VyaWVzIHVzaW5nIFByb3BoZXQNCiAgDQogIDIuIFVzaW5nIHJlZ3Jlc3NvcnMgc3VwcGxpZWQgdmlhIHRoZSBwcmVwcm9jZXNzaW5nIHJlY2lwZSAocmVtZW1iZXIgb3VyIHJlY2lwZSBnZW5lcmF0ZWQgNDUgbmV3IGZlYXR1cmVzKSwgYW5kIHJlZ3Jlc3NpbmcgdGhlIFByb3BoZXQgUmVzaWR1YWxzIHdpdGggdGhlIFhHQm9vc3QgbW9kZWwNCg0KV2UgY2FuIHNldCB0aGUgbW9kZWwgdXAgdXNpbmcgYSB3b3JrZmxvdyBqdXN0IGxpa2Ugd2l0aCB0aGUgbWFjaGluZSBsZWFybmluZyBhbGdvcml0aG1zLg0KDQpgYGB7cn0NCm1vZGVsX3NwZWNfcHJvcGhldF9ib29zdCA8LSBwcm9waGV0X2Jvb3N0KCkgJT4lIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2V0X2VuZ2luZSgicHJvcGhldF94Z2Jvb3N0IiwgeWVhcmx5LnNlYXNvbmFsaXR5ID0gVFJVRSkNCg0Kd29ya2Zsb3dfZml0X3Byb3BoZXRfYm9vc3QgPC0gd29ya2Zsb3coKSAlPiUgDQogIGFkZF9tb2RlbChtb2RlbF9zcGVjX3Byb3BoZXRfYm9vc3QpICU+JSANCiAgYWRkX3JlY2lwZShyZWNpcGVfc3BlYykgJT4lIA0KICBmaXQodHJhaW5pbmcoc3BsaXRzKSkNCg0KDQpgYGANCg0KDQojIyBNb2RlbHRpbWUgVGFibGUNCg0KYGBge3J9DQptb2RlbF90YWJsZSA8LSBtb2RlbHRpbWVfdGFibGUoDQogICAgICAgICAgICAgICAgICBtb2RlbF9maXRfYXJpbWEsDQogICAgICAgICAgICAgICAgICBtb2RlbF9maXRfcHJvcGhldCwNCiAgICAgICAgICAgICAgICAgIHdvcmtmbG93X2ZpdF9nbG1uZXQsDQogICAgICAgICAgICAgICAgICB3b3JrZmxvd19maXRfcmYsDQogICAgICAgICAgICAgICAgICB3b3JrZmxvd19maXRfcHJvcGhldF9ib29zdA0KICAgICAgICAgICAgICAgICAgKQ0KDQptb2RlbF90YWJsZQ0KYGBgDQoNCg0KIyMgQ2xhaWJyYXRpb24NCg0KYGBge3J9DQpjYWxpYnJhdGlvbl90YWJsZSA8LSBtb2RlbF90YWJsZSAlPiUgDQogIG1vZGVsdGltZV9jYWxpYnJhdGUodGVzdGluZyhzcGxpdHMpKQ0KDQpjYWxpYnJhdGlvbl90YWJsZQ0KYGBgDQoNCiMjIEZvcmVjYXN0aW5nDQoNCmBgYHtyfQ0KbGlicmFyeShwbG90bHkpDQpgYGANCg0KDQpgYGB7cn0NCmNhbGlicmF0aW9uX3RhYmxlICU+JSANCiAgbW9kZWx0aW1lX2ZvcmVjYXN0KGFjdHVhbF9kYXRhID0gYmlrZV90cmFuc2FjdGlvbnNfdGJsKSAlPiUgDQogIHBsb3RfbW9kZWx0aW1lX2ZvcmVjYXN0KCkgJT4lIA0KICBsYXlvdXQobGVnZW5kID0gbGlzdCh4ID0gLTAuNSwgeSA9IDEuMCkpDQoNCiAgIyBmb3IgbW9yZSBsZWdlbmQgcG9zaXRpb24gb3B0aW9ucyBjaGVjayBvdXQgaHR0cHM6Ly9wbG90bHkuY29tL3IvbGVnZW5kLw0KDQpgYGANCg0KDQpgYGB7cn0NClBfcmVzdWx0czEgPC0gY2FsaWJyYXRpb25fdGFibGUgJT4lIA0KICBtb2RlbHRpbWVfZm9yZWNhc3QoYWN0dWFsX2RhdGEgPSBiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JSANCiAgcGxvdF9tb2RlbHRpbWVfZm9yZWNhc3QoKSAlPiUgDQogIGxheW91dChsZWdlbmQgPSBsaXN0KG9yaWVudGF0aW9uID0gImgiLCB5ID0gLTAuMikpDQoNClBfcmVzdWx0czENCmBgYA0KDQpgYGB7cn0NCmxpYnJhcnkoaHRtbHdpZGdldHMpDQpgYGANCg0KYGBge3J9DQpodG1sd2lkZ2V0czo6c2F2ZVdpZGdldChQX3Jlc3VsdHMxLCAiZm9yZWNhc3RpbmdfcmVzdWx0cy5odG1sIikNCmBgYA0KDQoNCiMjIEFjY3VyYWN5IChUZXN0aW5nIHNldCkNCg0KYGBge3J9DQpjYWxpYnJhdGlvbl90YWJsZSAlPiUgDQogIG1vZGVsdGltZV9hY2N1cmFjeSgpICU+JSANCiAgdGFibGVfbW9kZWx0aW1lX2FjY3VyYWN5KC5pbnRlcmFjdGl2ZSA9IEZBTFNFKQ0KYGBgDQoNCg0KIyMgUmVmaXQgYW5kIEZvcmVjYXN0IEZvcndhcmQNCg0KYGBge3IsIG91dC53aWR0aD0gIjEwMCUifQ0KY2FsaWJyYXRpb25fdGFibGUgJT4lIA0KICANCiAgIyBSZW1vdmUgYXJpbWEgbW9kZWwgd2l0aCBsb3cgYWNjdXJhY3kNCiAgZmlsdGVyKC5tb2RlbF9pZCAhPTEpICU+JSANCiAgDQogICMgUmVmaXQgJiBGb3JlY2FzdCBGb3J3YXJkDQogIG1vZGVsdGltZV9yZWZpdChiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JSANCiAgbW9kZWx0aW1lX2ZvcmVjYXN0KGggPSAiMTIgbW9udGhzIiwgYWN0dWFsX2RhdGEgPSBiaWtlX3RyYW5zYWN0aW9uc190YmwpICU+JQ0KICBwbG90X21vZGVsdGltZV9mb3JlY2FzdCgpICU+JSANCiAgbGF5b3V0KGxlZ2VuZCA9IGxpc3Qob3JpZW50YXRpb24gPSAiaCIsIHkgPSAtMC4yKSkNCmBgYA0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0KDQoNCg0K